# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymysql
from scrapy.utils.project import get_project_settings


class KknetPipeline(object):
    def process_item(self, item, spider):
        with open('./dialoge.tsv', 'a', encoding='utf-8') as fp:
            dialoge = item['dialoge']
            text = '\n'.join(dialoge).strip()
            if len(text) < 10:
                return item
            fp.writelines('\n'.join(dialoge))
            fp.writelines('\n')
        return item


class KknetMysqlPipeline(object):

    def __init__(self):
        settings = get_project_settings()  # 获取settings配置数据
        self.db = settings['MYSQL_DBNAME'],
        self.host = settings['MYSQL_HOST'],
        self.port = settings['MYSQL_PORT']
        self.user = settings['MYSQL_USER'],
        self.password = settings['MYSQL_PASSWD'],

    def connectDatabase(self):
        con = pymysql.connect(
            host=self.host,
            port=self.port,
            user=self.user,
            password=self.password,
            db=self.db,
            charset='utf-8'
        )
        return con

    def process_item(self, item, spider):
        self._insert_item(item)
        return item

    def _insert_item(self, item):
        sql = "insert into `corpus`(`english`,`chinese`) values(%s,%s)"
        params = (item['english'], item['chinese'])

        self.db_cur.execute(sql, params)
